/**
* This file is part of ORB-SLAM3
*
* Copyright (C) 2017-2020 Carlos Campos, Richard Elvira, Juan J. Gómez Rodríguez, José M.M. Montiel and Juan D. Tardós, University of Zaragoza.
* Copyright (C) 2014-2016 Raúl Mur-Artal, José M.M. Montiel and Juan D. Tardós, University of Zaragoza.
*
* ORB-SLAM3 is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* ORB-SLAM3 is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with ORB-SLAM3.
* If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef KEYFRAME_H
#define KEYFRAME_H

#include "MapPoint.h"
#include "Thirdparty/DBoW2/DBoW2/BowVector.h"
#include "Thirdparty/DBoW2/DBoW2/FeatureVector.h"
#include "ORBVocabulary.h"
#include "ORBextractor.h"
#include "Frame.h"
#include "KeyFrameDatabase.h"
#include "ImuTypes.h"

#include "GeometricCamera.h"

#include <mutex>

#include <boost/serialization/base_object.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/map.hpp>

namespace ORB_SLAM3
{

class Map;
class MapPoint;
class Frame;
class KeyFrameDatabase;

class GeometricCamera;

class KeyFrame
{

    template <class Archive>
    void serializeMatrix(Archive &ar, cv::Mat &mat, const unsigned int version)
    {
        int cols, rows, type;
        bool continuous;

        if (Archive::is_saving::value)
        {
            cols = mat.cols;
            rows = mat.rows;
            type = mat.type();
            continuous = mat.isContinuous();
        }

        ar &cols &rows &type &continuous;

        if (Archive::is_loading::value)
            mat.create(rows, cols, type);

        if (continuous)
        {
            const unsigned int data_size = rows * cols * mat.elemSize();
            ar &boost::serialization::make_array(mat.ptr(), data_size);
        }
        else
        {
            const unsigned int row_size = cols * mat.elemSize();
            for (int i = 0; i < rows; i++)
            {
                ar &boost::serialization::make_array(mat.ptr(i), row_size);
            }
        }
    }

    template <class Archive>
    void serializeMatrix(Archive &ar, const cv::Mat &mat, const unsigned int version)
    {
        cv::Mat matAux = mat;

        serializeMatrix(ar, matAux, version);

        if (Archive::is_loading::value)
        {
            cv::Mat *ptr;
            ptr = (cv::Mat *)(&mat);
            *ptr = matAux;
        }
    }

    friend class boost::serialization::access;
    template <class Archive>
    void serializeVectorKeyPoints(Archive &ar, const vector<cv::KeyPoint> &vKP, const unsigned int version)
    {
        int NumEl;

        if (Archive::is_saving::value)
        {
            NumEl = vKP.size();
        }

        ar &NumEl;

        vector<cv::KeyPoint> vKPaux = vKP;
        if (Archive::is_loading::value)
            vKPaux.reserve(NumEl);

        for (int i = 0; i < NumEl; ++i)
        {
            cv::KeyPoint KPi;

            if (Archive::is_loading::value)
                KPi = cv::KeyPoint();

            if (Archive::is_saving::value)
                KPi = vKPaux[i];

            ar &KPi.angle;
            ar &KPi.response;
            ar &KPi.size;
            ar &KPi.pt.x;
            ar &KPi.pt.y;
            ar &KPi.class_id;
            ar &KPi.octave;

            if (Archive::is_loading::value)
                vKPaux.push_back(KPi);
        }

        if (Archive::is_loading::value)
        {
            vector<cv::KeyPoint> *ptr;
            ptr = (vector<cv::KeyPoint> *)(&vKP);
            *ptr = vKPaux;
        }
    }

    template <class Archive>
    void serialize(Archive &ar, const unsigned int version)
    {
        ar &mnId;
        ar &const_cast<long unsigned int &>(mnFrameId);
        ar &const_cast<double &>(mTimeStamp);
        // Grid
        ar &const_cast<int &>(mnGridCols);
        ar &const_cast<int &>(mnGridRows);
        ar &const_cast<float &>(mfGridElementWidthInv);
        ar &const_cast<float &>(mfGridElementHeightInv);
        // Variables of tracking
        ar &mnTrackReferenceForFrame;
        ar &mnFuseTargetForKF;
        // Variables of local mapping
        ar &mnBALocalForKF;
        ar &mnBAFixedForKF;
        ar &mnNumberOfOpt;
        // Variables used by KeyFrameDatabase
        ar &mnLoopQuery;
        ar &mnLoopWords;
        ar &mLoopScore;
        ar &mnRelocQuery;
        ar &mnRelocWords;
        ar &mRelocScore;
        ar &mnMergeQuery;
        ar &mnMergeWords;
        ar &mMergeScore;
        ar &mnPlaceRecognitionQuery;
        ar &mnPlaceRecognitionWords;
        ar &mPlaceRecognitionScore;
        ar &mbCurrentPlaceRecognition;
        // Variables of loop closing
        serializeMatrix(ar, mTcwGBA, version);
        serializeMatrix(ar, mTcwBefGBA, version);
        serializeMatrix(ar, mVwbGBA, version);
        serializeMatrix(ar, mVwbBefGBA, version);
        ar &mBiasGBA;
        ar &mnBAGlobalForKF;
        // Variables of Merging
        serializeMatrix(ar, mTcwMerge, version);
        serializeMatrix(ar, mTcwBefMerge, version);
        serializeMatrix(ar, mTwcBefMerge, version);
        serializeMatrix(ar, mVwbMerge, version);
        serializeMatrix(ar, mVwbBefMerge, version);
        ar &mBiasMerge;
        ar &mnMergeCorrectedForKF;
        ar &mnMergeForKF;
        ar &mfScaleMerge;
        ar &mnBALocalForMerge;
        // Scale
        ar &mfScale;
        // Calibration parameters
        ar &const_cast<float &>(fx);
        ar &const_cast<float &>(fy);
        ar &const_cast<float &>(invfx);
        ar &const_cast<float &>(invfy);
        ar &const_cast<float &>(cx);
        ar &const_cast<float &>(cy);
        ar &const_cast<float &>(mbf);
        ar &const_cast<float &>(mb);
        ar &const_cast<float &>(mThDepth);
        serializeMatrix(ar, mDistCoef, version);
        // Number of Keypoints
        ar &const_cast<int &>(N);
        // KeyPoints
        serializeVectorKeyPoints(ar, mvKeys, version);
        serializeVectorKeyPoints(ar, mvKeysUn, version);
        ar &const_cast<vector<float> &>(mvuRight);
        ar &const_cast<vector<float> &>(mvDepth);
        serializeMatrix(ar, mDescriptors, version);
        // BOW
        ar &mBowVec;
        ar &mFeatVec;
        // Pose relative to parent
        serializeMatrix(ar, mTcp, version);
        // Scale
        ar &const_cast<int &>(mnScaleLevels);
        ar &const_cast<float &>(mfScaleFactor);
        ar &const_cast<float &>(mfLogScaleFactor);
        ar &const_cast<vector<float> &>(mvScaleFactors);
        ar &const_cast<vector<float> &>(mvLevelSigma2);
        ar &const_cast<vector<float> &>(mvInvLevelSigma2);
        // Image bounds and calibration
        ar &const_cast<int &>(mnMinX);
        ar &const_cast<int &>(mnMinY);
        ar &const_cast<int &>(mnMaxX);
        ar &const_cast<int &>(mnMaxY);
        serializeMatrix(ar, mK, version);
        // Pose
        serializeMatrix(ar, Tcw, version);
        // MapPointsId associated to keypoints
        ar &mvBackupMapPointsId;
        // Grid
        ar &mGrid;
        // Connected KeyFrameWeight
        ar &mBackupConnectedKeyFrameIdWeights;
        // Spanning Tree and Loop Edges
        ar &mbFirstConnection;
        ar &mBackupParentId;
        ar &mvBackupChildrensId;
        ar &mvBackupLoopEdgesId;
        ar &mvBackupMergeEdgesId;
        // Bad flags
        ar &mbNotErase;
        ar &mbToBeErased;
        ar &mbBad;

        ar &mHalfBaseline;

        // Camera variables
        ar &mnBackupIdCamera;
        ar &mnBackupIdCamera2;

        // Fisheye variables
        /*ar & mvLeftToRightMatch;
    ar & mvRightToLeftMatch;
    ar & NLeft;
    ar & NRight;
    serializeMatrix(ar, mTlr, version);
    //serializeMatrix(ar, mTrl, version);
    serializeVectorKeyPoints(ar, mvKeysRight, version);
    ar & mGridRight;

    // Inertial variables
    ar & mImuBias;
    ar & mBackupImuPreintegrated;
    ar & mImuCalib;
    ar & mBackupPrevKFId;
    ar & mBackupNextKFId;
    ar & bImu;
    serializeMatrix(ar, Vw, version);
    serializeMatrix(ar, Owb, version);*/
    }

public:
    KeyFrame();
    KeyFrame(Frame &F, Map *pMap, KeyFrameDatabase *pKFDB);

    // Pose functions
    void SetPose(const cv::Mat &Tcw);
    void SetVelocity(const cv::Mat &Vw_);

    cv::Mat GetPose();
    cv::Mat GetPoseInverse();
    cv::Mat GetCameraCenter();
    cv::Mat GetImuPosition();
    cv::Mat GetImuRotation();
    cv::Mat GetImuPose();
    cv::Mat GetStereoCenter();
    cv::Mat GetRotation();
    cv::Mat GetTranslation();
    cv::Mat GetVelocity();

    // Bag of Words Representation
    void ComputeBoW();

    // Covisibility graph functions
    void AddConnection(KeyFrame *pKF, const int &weight);
    void EraseConnection(KeyFrame *pKF);

    void UpdateConnections(bool upParent = true);
    void UpdateBestCovisibles();
    std::set<KeyFrame *> GetConnectedKeyFrames();
    std::vector<KeyFrame *> GetVectorCovisibleKeyFrames();
    std::vector<KeyFrame *> GetBestCovisibilityKeyFrames(const int &N);
    std::vector<KeyFrame *> GetCovisiblesByWeight(const int &w);
    int GetWeight(KeyFrame *pKF);

    // Spanning tree functions
    void AddChild(KeyFrame *pKF);
    void EraseChild(KeyFrame *pKF);
    void ChangeParent(KeyFrame *pKF);
    std::set<KeyFrame *> GetChilds();
    KeyFrame *GetParent();
    bool hasChild(KeyFrame *pKF);
    void SetFirstConnection(bool bFirst);

    // Loop Edges
    void AddLoopEdge(KeyFrame *pKF);
    std::set<KeyFrame *> GetLoopEdges();

    // Merge Edges
    void AddMergeEdge(KeyFrame *pKF);
    set<KeyFrame *> GetMergeEdges();

    // MapPoint observation functions
    int GetNumberMPs();
    void AddMapPoint(MapPoint *pMP, const size_t &idx);
    void EraseMapPointMatch(const int &idx);
    void EraseMapPointMatch(MapPoint *pMP);
    void ReplaceMapPointMatch(const int &idx, MapPoint *pMP);
    std::set<MapPoint *> GetMapPoints();
    std::vector<MapPoint *> GetMapPointMatches();
    int TrackedMapPoints(const int &minObs);
    MapPoint *GetMapPoint(const size_t &idx);

    // KeyPoint functions
    std::vector<size_t> GetFeaturesInArea(const float &x, const float &y, const float &r, const bool bRight = false) const;
    cv::Mat UnprojectStereo(int i);

    // Image
    bool IsInImage(const float &x, const float &y) const;

    // Enable/Disable bad flag changes
    void SetNotErase();
    void SetErase();

    // Set/check bad flag
    void SetBadFlag();
    bool isBad();

    // Compute Scene Depth (q=2 median). Used in monocular.
    float ComputeSceneMedianDepth(const int q);

    static bool weightComp(int a, int b)
    {
        return a > b;
    }

    static bool lId(KeyFrame *pKF1, KeyFrame *pKF2)
    {
        return pKF1->mnId < pKF2->mnId;
    }

    Map *GetMap();
    void UpdateMap(Map *pMap);

    void SetNewBias(const IMU::Bias &b);
    cv::Mat GetGyroBias();
    cv::Mat GetAccBias();
    IMU::Bias GetImuBias();

    bool ProjectPointDistort(MapPoint *pMP, cv::Point2f &kp, float &u, float &v);
    bool ProjectPointUnDistort(MapPoint *pMP, cv::Point2f &kp, float &u, float &v);

    void PreSave(set<KeyFrame *> &spKF, set<MapPoint *> &spMP, set<GeometricCamera *> &spCam);
    void PostLoad(map<long unsigned int, KeyFrame *> &mpKFid, map<long unsigned int, MapPoint *> &mpMPid, map<unsigned int, GeometricCamera *> &mpCamId);

    void SetORBVocabulary(ORBVocabulary *pORBVoc);
    void SetKeyFrameDatabase(KeyFrameDatabase *pKFDB);

    bool bImu;

    // The following variables are accesed from only 1 thread or never change (no mutex needed).
public:
    static long unsigned int nNextId;
    long unsigned int mnId;
    const long unsigned int mnFrameId;

    const double mTimeStamp;

    // Grid (to speed up feature matching)
    const int mnGridCols;
    const int mnGridRows;
    const float mfGridElementWidthInv;
    const float mfGridElementHeightInv;

    // Variables used by the tracking
    long unsigned int mnTrackReferenceForFrame; // mnTrackReferenceForFrame防止重复添加局部关键帧
    long unsigned int mnFuseTargetForKF;

    // Variables used by the local mapping
    long unsigned int mnBALocalForKF;
    long unsigned int mnBAFixedForKF;

    //Number of optimizations by BA(amount of iterations in BA)
    long unsigned int mnNumberOfOpt;

    // Variables used by the keyframe database
    long unsigned int mnLoopQuery;
    int mnLoopWords;
    float mLoopScore;
    long unsigned int mnRelocQuery;
    int mnRelocWords;
    float mRelocScore;
    long unsigned int mnMergeQuery;
    int mnMergeWords;
    float mMergeScore;
    long unsigned int mnPlaceRecognitionQuery;
    int mnPlaceRecognitionWords;
    float mPlaceRecognitionScore;

    bool mbCurrentPlaceRecognition;

    // Variables used by loop closing
    cv::Mat mTcwGBA;
    cv::Mat mTcwBefGBA;
    cv::Mat mVwbGBA;
    cv::Mat mVwbBefGBA;
    IMU::Bias mBiasGBA;
    long unsigned int mnBAGlobalForKF;

    // Variables used by merging
    cv::Mat mTcwMerge;
    cv::Mat mTcwBefMerge;
    cv::Mat mTwcBefMerge;
    cv::Mat mVwbMerge;
    cv::Mat mVwbBefMerge;
    IMU::Bias mBiasMerge;
    long unsigned int mnMergeCorrectedForKF;
    long unsigned int mnMergeForKF;
    float mfScaleMerge;
    long unsigned int mnBALocalForMerge;

    float mfScale;

    // Calibration parameters
    const float fx, fy, cx, cy, invfx, invfy, mbf, mb, mThDepth;
    cv::Mat mDistCoef;

    // Number of KeyPoints
    const int N;

    // KeyPoints, stereo coordinate and descriptors (all associated by an index)
    const std::vector<cv::KeyPoint> mvKeys;
    const std::vector<cv::KeyPoint> mvKeysUn;
    const std::vector<float> mvuRight; // negative value for monocular points
    const std::vector<float> mvDepth;  // negative value for monocular points
    const cv::Mat mDescriptors;

    //BoW
    DBoW2::BowVector mBowVec;
    DBoW2::FeatureVector mFeatVec;

    // Pose relative to parent (this is computed when bad flag is activated)
    cv::Mat mTcp;

    // Scale
    const int mnScaleLevels;
    const float mfScaleFactor;
    const float mfLogScaleFactor;
    const std::vector<float> mvScaleFactors;
    const std::vector<float> mvLevelSigma2;
    const std::vector<float> mvInvLevelSigma2;

    // Image bounds and calibration
    const int mnMinX;
    const int mnMinY;
    const int mnMaxX;
    const int mnMaxY;
    const cv::Mat mK;

    // Preintegrated IMU measurements from previous keyframe
    KeyFrame *mPrevKF;
    KeyFrame *mNextKF;

    IMU::Preintegrated *mpImuPreintegrated;
    IMU::Calib mImuCalib;

    unsigned int mnOriginMapId;

    string mNameFile;

    int mnDataset;

    std::vector<KeyFrame *> mvpLoopCandKFs;
    std::vector<KeyFrame *> mvpMergeCandKFs;

    bool mbHasHessian;
    cv::Mat mHessianPose;

    // The following variables need to be accessed trough a mutex to be thread safe.
protected:
    // SE3 Pose and camera center
    cv::Mat Tcw;
    cv::Mat Twc;
    cv::Mat Ow;
    cv::Mat Cw; // Stereo middel point. Only for visualization

    // IMU position
    cv::Mat Owb;

    // Velocity (Only used for inertial SLAM)
    cv::Mat Vw;

    // Imu bias
    IMU::Bias mImuBias;

    // MapPoints associated to keypoints
    std::vector<MapPoint *> mvpMapPoints;
    // For save relation without pointer, this is necessary for save/load function
    std::vector<long long int> mvBackupMapPointsId;

    // BoW
    KeyFrameDatabase *mpKeyFrameDB;
    ORBVocabulary *mpORBvocabulary;

    // Grid over the image to speed up feature matching
    std::vector<std::vector<std::vector<size_t>>> mGrid;

    std::map<KeyFrame *, int> mConnectedKeyFrameWeights; // 存放与之相连的关键帧与权重（也就是MP点数）
    std::vector<KeyFrame *> mvpOrderedConnectedKeyFrames;
    std::vector<int> mvOrderedWeights;
    // For save relation without pointer, this is necessary for save/load function
    std::map<long unsigned int, int> mBackupConnectedKeyFrameIdWeights;

    // Spanning Tree and Loop Edges
    bool mbFirstConnection;
    KeyFrame *mpParent;
    std::set<KeyFrame *> mspChildrens;
    std::set<KeyFrame *> mspLoopEdges;
    std::set<KeyFrame *> mspMergeEdges;
    // For save relation without pointer, this is necessary for save/load function
    long long int mBackupParentId;
    std::vector<long unsigned int> mvBackupChildrensId;
    std::vector<long unsigned int> mvBackupLoopEdgesId;
    std::vector<long unsigned int> mvBackupMergeEdgesId;

    // Bad flags
    bool mbNotErase;
    bool mbToBeErased;
    bool mbBad;

    float mHalfBaseline; // Only for visualization

    Map *mpMap;

    std::mutex mMutexPose; // for pose, velocity and biases
    std::mutex mMutexConnections;
    std::mutex mMutexFeatures;
    std::mutex mMutexMap;

    // Backup variables for inertial
    long long int mBackupPrevKFId;
    long long int mBackupNextKFId;
    IMU::Preintegrated mBackupImuPreintegrated;

    // Backup for Cameras
    unsigned int mnBackupIdCamera, mnBackupIdCamera2;

public:
    GeometricCamera *mpCamera, *mpCamera2;

    //Indexes of stereo observations correspondences
    std::vector<int> mvLeftToRightMatch, mvRightToLeftMatch;

    //Transformation matrix between cameras in stereo fisheye
    cv::Mat mTlr;
    cv::Mat mTrl;

    //KeyPoints in the right image (for stereo fisheye, coordinates are needed)
    const std::vector<cv::KeyPoint> mvKeysRight;

    const int NLeft, NRight;

    std::vector<std::vector<std::vector<size_t>>> mGridRight;

    cv::Mat GetRightPose();
    cv::Mat GetRightPoseInverse();
    cv::Mat GetRightPoseInverseH();
    cv::Mat GetRightCameraCenter();
    cv::Mat GetRightRotation();
    cv::Mat GetRightTranslation();

    cv::Mat imgLeft, imgRight; //TODO Backup??

    void PrintPointDistribution()
    {
        int left = 0, right = 0;
        int Nlim = (NLeft != -1) ? NLeft : N;
        for (int i = 0; i < N; i++)
        {
            if (mvpMapPoints[i])
            {
                if (i < Nlim)
                    left++;
                else
                    right++;
            }
        }
        cout << "Point distribution in KeyFrame: left-> " << left << " --- right-> " << right << endl;
    }
};

} // namespace ORB_SLAM3

#endif // KEYFRAME_H
